/*
 * Copyright (C) 2010  Paulo Cesar Pereira de Andrade.
 *
 * This is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * This software is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * Authors:
 *	Paulo Cesar Pereira de Andrade
 */

#include "exl.h"

#include <stddef.h>
#include <lightning.h>
#include <sys/mman.h>

#if HAVE_EXECINFO_H
#  include <execinfo.h>
#endif

#define JIT_SETJMP		1

#if !JIT_SETJMP
#  define JIT_REENTER		1
#endif

/* Not defined elsewhere to avoid confusion */
#if __WORDSIZE == 32
#  define t_word	t_int32
#else
#  define t_word	t_int64
#endif

/* implicit value in register
 * if set, the thread implicit value is probably invalid */
#define lit_sync_register	1

/* implicit value in thread state
 * used in conjunction with lit_sync_register, otherwise, the thread state
 * *is* the implicit value */
#define lit_sync_implicit	2

/* implicit value in register has been zero or sign extended
 * used when the implicit value fits in a 32 bit register, and only
 * meaningul when need two integer registers to represent the implicit value */
#define lit_sync_int_ext	4

/* implicit value has been converted to/from float/double
 * meaningul when loading/storing different float type values */
#define lit_sync_float_ext	8

/* implicit int value has been converted to a float/double
 * it means the implicit value is an integer type, and the float
 * register has been converted, or vice-versa, that is, implicit is
 * a float type, but integer register has been truncated */
#define lit_sync_int_float	16
#define lit_sync_int_double	32
#define lit_sync_long_float	64
#define lit_sync_long_double	128

/*
 * Types
 */
typedef eobject_t (*jit_t)(ethread_t *);
#define call_jit(jit, thread)		(*jit)(thread)

/*
 * Prototypes
 */
#if !JIT_REENTER && !JIT_SETJMP
static ethread_t *
get_thread_self(void);
#endif

static void
call_v_v(void(*v_v)(void));

static void
call_v_i(void(*v_i)(eint32_t), eint32_t arg0);

static void
call_p_i(eobject_t(*p_i)(eint32_t), eint32_t arg0);

static void
call_v_i_i(void(*v_i_i)(eint32_t, eint32_t), eint32_t arg0, eint32_t arg1);

static void
call_p_i_i(eobject_t(*p_i_i)(eint32_t, eint32_t), eint32_t arg0, eint32_t arg1);

static void
call_v_i_i_i(void(*v_i_i_i)(eint32_t, eint32_t, eint32_t),
	     eint32_t arg0, eint32_t arg1, eint32_t arg2);

static void
call_v_i_i_i_i(void(*v_i_i_i_i)(eint32_t, eint32_t, eint32_t, eint32_t),
	       eint32_t arg0, eint32_t arg1, eint32_t arg2, eint32_t arg3);

static eword_t
allocate_forward_label(elabel_t *label);

static void
record_forward_patch(eword_t offset, eword_t label);

static void
emit_branch_forward_patch(ecode_t code, eword_t label);

static void
emit_branch_finalize(ecode_t code, elabel_t *label);

static void
emit_branch(ecode_t code, elabel_t *label);

static ecode_t
switches(ehash_t *hash, eint32_t offset);

static void
record_table_patch(eword_t offset);

static void
emit_table_branch(ehash_t *hash, eint32_t offset);

static void
vswitch_patch(evswitch_t *table);

static void
hswitch_patch(ehswitch_t *table);

static void
table_patch(eobject_t object);

static void
record_forward_call(eword_t offset);

static eobject_t
emit_frame(void);

static void
emit_call(eint32_t offset);

static void
emit_method_call(eint32_t offset, eint32_t type);

static void
record_forward_method(eword_t type, eword_t offset);

static void
emit_explicit_call(eint32_t offset, eint32_t type);

static void
emit_enter(east_node_t *node);

static void
emit_vnewi(east_node_t *node);

static void
emit_try(east_list_t *list);

static void
emit_stackref(eint32_t offset, eint32_t type);

#define PROTO		1
#include "ethunder.c"
#include "erain.c"
#undef PROTO

/*
 * Initialization
 */
eobject_t		  ethread_exit_code;

static jit_insn		 *the_code;
static eobject_t	  the_end;
static elit_node_t	**lit_base;
static elit_node_t	**lit_tail;
static eint32_t		 *lit_stack;
static eint32_t		  lit_type;
static eint32_t		  lit_sync;
static eobject_t	 *labels;
static eword_t		 *patches;
static eword_t		 *tables;
static eobject_t	 *calls;
static eword_t		 *methods;
static evector_t	 *root_vector;
static evector_t	 *base_vector;
static evector_t	 *stack_vector;
static evector_t	 *label_vector;
static evector_t	 *patch_vector;
static evector_t	 *table_vector;
static evector_t	 *call_vector;
static evector_t	 *method_vector;
static char		 *exceptions[] = {
    "nothing",
    "out of memory",
    "floating point error",
    "out of bounds",
    "null dereference",
    "type mismatch",
    "input output error",
    "invalid argument",
    "not a number",
    "not an integer",
    "not a real number",
    "not a rational number",
    "not a 32 bits integer",
    "not a finite real number",
};

/*
 * Implementation
 */
void
init_lightning(void)
{
    eobject_t	*root;
    eword_t	 thread;

#if defined(jit_push_pop_p)
    jit_flags.push_pop = 0;
#endif

#if DISASSEMBLER
    if (cfg_verbose > 1)
	init_debug();
#endif

    evector((eobject_t *)&root_vector, t_void, 8, 1);
    eadd_root((eobject_t *)&root_vector);
    root = root_vector->ptr;

    evector((eobject_t *)&base_vector, t_void, 4, 1);
    root[0] = base_vector;
    enew_object((eobject_t *)&lit_tail, t_void, sizeof(eobject_t) * 4);
    root[1] = lit_tail;
    evector((eobject_t *)&stack_vector, t_int32, 8, 1);
    root[2] = stack_vector;
    lit_base = base_vector->ptr;
    lit_stack = stack_vector->ptr;

    /* patch offsets for forward defined functions */
    evector((eobject_t *)&call_vector, t_word, 32, 1);
    root[3] = call_vector;
    calls = call_vector->ptr;

    /* patch offsets for forward defined methods */
    evector((eobject_t *)&method_vector, t_word, 32, 1);
    root[4] = method_vector;
    methods = method_vector->ptr;

    /* function scope label addresses */
    evector((eobject_t *)&label_vector, t_word, 32, 1);
    root[5] = label_vector;
    labels = label_vector->ptr;
    /* forward jump references */
    evector((eobject_t *)&patch_vector, t_word, 32, 1);
    root[6] = patch_vector;
    patches = patch_vector->ptr;
    /* switch table references */
    evector((eobject_t *)&table_vector, t_word, 8, 1);
    root[7] = table_vector;
    tables = table_vector->ptr;

    /* FIXME need to keep track of offsets of jumps and movs for every
     * function until finish_lightning() if needs to reallocate the
     * buffer and if the base address may change (mremap may do that?)
     * FIXME also, need to actually check if jit generation overruns
     * this buffer...
     */

#if 1
    /* Initialize at 16Mb mapping */
    the_code = mmap(null, 16 * 1024 * 1024, PROT_EXEC | PROT_READ | PROT_WRITE,
		    MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
    if (the_code == MAP_FAILED)
	eerror("mmap failed (%s)", strerror(errno));
#else
    posix_memalign((void **)&the_code, 4096, 16 * 1024 * 1024);
#endif
    /* make it easier to know if a pointer is inside the code */
    the_end = (eint8_t *)the_code + 16 * 1024 * 1024;

    (void)jit_set_ip(the_code);

    jit_prolog(1);
    thread = jit_arg_p();
    jit_getarg_p(JIT_V0, thread);

    /* this should be the safest path, only possible problem is possible
     * portability issues with __sigsetjmp identifier, called by the
     * sigsetjmp macro in glibc */
#if JIT_SETJMP
    {
	eobject_t	label_segv;
	eobject_t	label_catch;
	eobject_t	label_except;
	eobject_t	label_execute;
	eobject_t	label_reenter;

	/* call sigsetjmp when starting thread */
	jit_prepare(2);
	{
	    jit_movi_i(JIT_R0, 1);
	    jit_pusharg_i(JIT_R0);

	    jit_addi_p(JIT_R0, JIT_V0, offsetof(ethread_t, env));
	    jit_pusharg_p(JIT_R0);
	}
	jit_finish(__sigsetjmp);

	jit_retval_i(JIT_R0);
	/* jump to check for sigsegv if coming from siglongjmp,
	 * that is, if sigsetjmp did not return zero */
	label_segv = jit_bnei_ui(jit_forward(), JIT_R0, 0);
	jit_prepare(1);
	{
	    /* release thread mutex */
	    (void)jit_movi_p(JIT_R0, &ethread_mutex);
	    jit_pusharg_p(JIT_R0);
	}
	jit_finish(pthread_mutex_unlock);
	/* jump to start of thread code */
	label_execute = jit_jmpi(jit_forward());

	jit_patch(label_segv);
	/* keep value in R1 */
	jit_ldxi_i(JIT_R1, JIT_V0, offsetof(ethread_t, except));
	/* jump to check try/catch if not coming from sigsegv signal handler */
	label_catch = jit_bnei_ui(jit_forward(), JIT_R1,
				  except_segmentation_violation);
	jit_prepare(2);
	{
	    (void)jit_movi_p(JIT_R0, "segmentation violation");
	    jit_pusharg_p(JIT_R0);

	    jit_ldxi_p(JIT_R0, JIT_V0, offsetof(ethread_t, eip));
	    jit_pusharg_p(JIT_R0);
	}
	jit_finish(eabort);

	jit_patch(label_catch);
	/* keep value in R2 */
	jit_ldxi_i(JIT_R2, JIT_V0, offsetof(ethread_t, tryoff));
	/* jump to reenter code if there is a handler, i.e.
	 * thread_self->tryoff >= 0 */
	label_reenter = jit_bgei_i(jit_forward(), JIT_R2, 0);
	/* jump to code to not print exception name if there isn't an
	 * entry in the exceptions string vector */
	label_except = jit_bgei_ui(jit_forward(), JIT_R1,
				   except_unhandled_exception);
	jit_prepare(2);
	{
	    jit_muli_i(JIT_R0, JIT_R1, sizeof(char *));
	    (void)jit_movi_p(JIT_R2, exceptions);
	    jit_ldxr_p(JIT_R0, JIT_R2, JIT_R0);
	    jit_pusharg_p(JIT_R0);
	    jit_ldxi_p(JIT_R0, JIT_V0, offsetof(ethread_t, eip));
	    jit_pusharg_p(JIT_R0);
	}
	jit_finish(eabort);

	jit_patch(label_except);
	jit_prepare(2);
	{
	    (void)jit_movi_p(JIT_R0, "unknown");
	    jit_pusharg_p(JIT_R0);
	    jit_ldxi_p(JIT_R0, JIT_V0, offsetof(ethread_t, eip));
	    jit_pusharg_p(JIT_R0);
	}
	jit_finish(eabort);

	/* reenter jit after exception */
	jit_patch(label_reenter);
	/* r0.t = t_int */
	jit_movi_i(JIT_R0, t_int);
	jit_stxi_i(offsetof(ethread_t, r0) + offsetof(evalue_t, t),
		   JIT_V0, JIT_R0);
	/* r0.v.i = thread->except */
#  if __WORDSIZE == 32
	jit_stxi_i(offsetof(ethread_t, r0) +
		   offsetof(evalue_t, v.i) + offsetof(edata_t, s.l),
		   JIT_V0, JIT_R1);
	jit_movi_i(JIT_R0, 0);
	jit_stxi_i(offsetof(ethread_t, r0) +
		   offsetof(evalue_t, v.i) + offsetof(edata_t, s.h),
		   JIT_V0, JIT_R0);
#  else
	jit_stxi_l(offsetof(ethread_t, r0) + offsetof(evalue_t, v.i),
		   JIT_V0, JIT_R1);
	jit_movi_i(JIT_R0, 0);
#  endif
	/* thread->except = except_nothing */
	jit_stxi_i(offsetof(ethread_t, except), JIT_V0, JIT_R0);

	/* jump to catch handler, in thread->ess[thread->tryoff + 3].v.o */
	jit_addi_i(JIT_R0, JIT_R2, 3);
	jit_muli_i(JIT_R0, JIT_R0, sizeof(evalue_t));
	jit_addi_i(JIT_R0, JIT_R0, offsetof(evalue_t, v.o));
	jit_ldxi_p(JIT_R1, JIT_V0, offsetof(ethread_t, ess));
	jit_ldxr_p(JIT_R0, JIT_R1, JIT_R0);
	jit_jmpr(JIT_R0);

	/* start from here if starting thread */
	jit_patch(label_execute);
    }
#endif

    /* jump to start of the thread routine, that should be just
     * after this jump opcode for the main thread */
    jit_ldxi_p(JIT_R0, JIT_V0, offsetof(ethread_t, eip));
    jit_jmpr(JIT_R0);
}

void
finish_lightning(void)
{
    eint32_t	 index;
    eword_t	 offset;
    erecord_t	*record;
    efunction_t	*lambda;
    esymbol_t	*symbol;

    for (offset = 0; offset < ecs_idx; offset++) {
	if (!ejit_pointer_p(ecs[offset])) {
	    symbol = ecs[offset];
	    lambda = symbol->value;
	    if (!efunction_p(lambda) || !lambda->defined)
		eerror("function '%p' not defined", symbol->name);
	    else
		ewarn("function '%p' defined but not used", symbol->name);
	    /* not compiled because it was not called or referenced */
	    ecs[offset] = null;
	}
    }

    /* since all methods are virtual, ensure all are defined */
    for (offset = 0; offset < ets_idx; offset++) {
	record = record_vector[offset];
	assert(offset == (record->type & ~t_pointer));
	if (record->methods) {
	    for (index = 0; index < record->methods->count; index++) {
		if (!ejit_pointer_p(ets[offset].methods[index].code)) {
		    symbol = ets[offset].methods[index].code;
		    lambda = symbol->value;
		    if (!efunction_p(lambda) || !lambda->defined)
			eerror("method '%p.%p' not defined",
			       record->name->name,
			       emethod_name(record, symbol));
		    /* this should have been already checked that is
		     * always correct, and since super has a lower
		     * type value, it already was processed */
		    ets[offset].methods[index].code =
			ets[record->super->type].methods[index].code;
		}
	    }
	}
    }

    /* patch forward calls */
    for (offset = 0; offset < call_vector->offset; offset += 2)
	jit_patch_at(calls[offset], ecs[(eword_t)calls[offset + 1]]);

    /* patch forward methods */
    for (offset = 0; offset < method_vector->offset; offset += 3)
	jit_patch_at((eobject_t)methods[offset], ets[methods[offset + 1]].
		     methods[methods[offset + 2]].code);

    erem_root((eobject_t *)&root_vector);

    /* thread exit trampoline */
    ethread_exit_code = (eobject_t)jit_get_label();
    (void)jit_calli(evm_exit);

    /* marker of end of code section, for backtrace information */
    the_end = (eobject_t)jit_get_label();

    jit_flush_code(the_code, jit_get_ip().ptr);

#if DISASSEMBLER
    if (cfg_verbose > 1)
	finish_debug();
#endif
}

eobject_t
evm(ethread_t *thread)
{
    jit_t	jit;

    if (thread != tmain) {
	evalue_t	*v;

	thread_self = thread;
	empcache_setup();
	for (v = thread->esp - 3; v >= thread->ess; v--) {
	    switch (v->t) {
		case t_mpz:	enew_mpz(&v->v.o, v->v.o);	break;
		case t_mpq:	enew_mpq(&v->v.o, v->v.o);	break;
		case t_mpr:	enew_mpr(&v->v.o, v->v.o);	break;
		case t_cqq:	enew_cqq(&v->v.o, v->v.o);	break;
		case t_mpc:	enew_mpc(&v->v.o, v->v.o);	break;
		default:					break;
	    }
	}
    }

    /* in case gc is running and will try to inspect this thread stack */
    ethreads_lock();
    thread->running = 1;
    thread->except = except_nothing;

#if !JIT_SETJMP
    /* arm signal exception handler and unlock threads linked list */
    if (sigsetjmp(thread->env, 1) == 0)
	ethreads_unlock();
    /* coming from signal handler or siglongjmp from builtin code */
    else if (thread->except == except_segmentation_violation)
	eabort((eint8_t *)thread->eip, "segmentation violation");
    else if (thread->tryoff < 0) {
	if ((euint32_t)thread->except < except_unhandled_exception)
	    eabort((eint8_t *)ip, exceptions[thread->except]);
	eabort((eint8_t *)thread->eip, "unknown");
    }
    else {
	thread->r0.t = t_int;
	thread->r0.v.i = thread->except;
	thread->except = except_nothing;

#  if JIT_REENTER
	thread->eip = thread->ess[thread->tryoff + 3].v.o;
#  else
	/* jump back into exception handler */
	jit = (jit_t)thread->ess[thread->tryoff + 3].v.o;
	return (call_jit(jit, thread));
#  endif
    }
#endif

    jit = (jit_t)the_code;

    return (call_jit(jit, thread));
}

void
esearch_instruction_pointer(void)
{
    eobject_t	eip;
#if HAVE_EXECINFO_H
    eint32_t	length;
    eint32_t	offset;
    eobject_t	frames[32];

    length = backtrace(frames, sizeof(frames) / sizeof(frames[0]));
    for (offset = 0; offset < length; offset++) {
	eip = frames[offset];
	if (eip >= (eobject_t)the_code && eip < the_end) {
	    thread_self->eip = eip;
	    break;
	}
    }
#else	/* this may still have problems if not compiled with
	 * -fno-omit-frame-pointer on i686 */
    /* __builtin_return_address() and __builtin_frame_address()
     * require an integer constant argument */
#  define search_instruction_pointer(number)				\
    do {								\
	eip = __builtin_return_address(number);				\
	if (eip == null || __builtin_frame_address(number) == null)	\
	    goto fail;							\
	if (eip >= (eobject_t)the_code && eip < the_end)		\
	    goto done;							\
    } while (0)
    search_instruction_pointer(0);
#  if !defined(__x86_64__)
    search_instruction_pointer(1);
    search_instruction_pointer(2);
    search_instruction_pointer(3);
    search_instruction_pointer(4);
    search_instruction_pointer(5);
    search_instruction_pointer(6);
    search_instruction_pointer(7);
    search_instruction_pointer(8);
    search_instruction_pointer(9);
    search_instruction_pointer(10);
    search_instruction_pointer(11);
    search_instruction_pointer(12);
    search_instruction_pointer(13);
    search_instruction_pointer(14);
    search_instruction_pointer(15);
    search_instruction_pointer(16);
    search_instruction_pointer(17);
    search_instruction_pointer(18);
    search_instruction_pointer(19);
    search_instruction_pointer(20);
    search_instruction_pointer(21);
    search_instruction_pointer(22);
    search_instruction_pointer(23);
    search_instruction_pointer(24);
    search_instruction_pointer(25);
    search_instruction_pointer(26);
    search_instruction_pointer(27);
    search_instruction_pointer(28);
    search_instruction_pointer(29);
    search_instruction_pointer(30);
    search_instruction_pointer(31);
    /* Greed is Good! (the more stack frames the better) */
#  endif
fail:
    /* will probably print a bogus backtrace with an early address */
    return;
done:
    thread_self->eip = eip;
#endif
}

ebool_t
ejit_pointer_p(eobject_t value)
{
    return ((eword_t)value >= (eword_t)the_code &&
	    (eword_t)value <  (eword_t)the_end);
}

#if !JIT_REENTER && !JIT_SETJMP
static ethread_t *
get_thread_self(void)
{
    return (thread_self);
}
#endif

static void
call_v_v(void(*v_v)(void))
{
    (void)jit_calli(v_v);
}

static void
call_v_i(void(*v_i)(eint32_t), eint32_t arg0)
{
    jit_prepare(1);
    jit_movi_i(JIT_R0, arg0);
    jit_pusharg_i(JIT_R0);
    jit_finish(v_i);
}

static void
call_p_i(eobject_t(*p_i)(eint32_t), eint32_t arg0)
{
    jit_prepare(1);
    jit_movi_i(JIT_R0, arg0);
    jit_pusharg_i(JIT_R0);
    jit_finish(p_i);
}

static void
call_v_i_i(void(*v_i_i)(eint32_t, eint32_t), eint32_t arg0, eint32_t arg1)
{
    jit_prepare(2);
    jit_movi_i(JIT_R0, arg1);
    jit_pusharg_i(JIT_R0);
    if (arg0 != arg1)
	jit_movi_i(JIT_R0, arg0);
    jit_pusharg_i(JIT_R0);
    jit_finish(v_i_i);
}

static void
call_p_i_i(eobject_t(*p_i_i)(eint32_t, eint32_t), eint32_t arg0, eint32_t arg1)
{
    jit_prepare(2);
    jit_movi_i(JIT_R0, arg1);
    jit_pusharg_i(JIT_R0);
    if (arg0 != arg1)
	jit_movi_i(JIT_R0, arg0);
    jit_pusharg_i(JIT_R0);
    jit_finish(p_i_i);
}

static void
call_v_i_i_i(void(*v_i_i_i)(eint32_t, eint32_t, eint32_t),
	     eint32_t arg0, eint32_t arg1, eint32_t arg2)
{
    jit_prepare(3);
    jit_movi_i(JIT_R0, arg2);
    jit_pusharg_i(JIT_R0);
    if (arg1 != arg2)
	jit_movi_i(JIT_R0, arg1);
    jit_pusharg_i(JIT_R0);
    if (arg0 != arg1)
	jit_movi_i(JIT_R0, arg0);
    jit_pusharg_i(JIT_R0);
    jit_finish(v_i_i_i);
}

static void
call_v_i_i_i_i(void(*v_i_i_i_i)(eint32_t, eint32_t, eint32_t, eint32_t),
	       eint32_t arg0, eint32_t arg1, eint32_t arg2, eint32_t arg3)
{
    jit_prepare(4);
    jit_movi_i(JIT_R0, arg3);
    jit_pusharg_i(JIT_R0);
    if (arg2 != arg3)
	jit_movi_i(JIT_R0, arg2);
    jit_pusharg_i(JIT_R0);
    if (arg1 != arg2)
	jit_movi_i(JIT_R0, arg1);
    jit_pusharg_i(JIT_R0);
    if (arg0 != arg1)
	jit_movi_i(JIT_R0, arg0);
    jit_pusharg_i(JIT_R0);
    jit_finish(v_i_i_i_i);
}

static eword_t
allocate_forward_label(elabel_t *label)
{
    eword_t	offset;

    /* skip first entry to be able to use a negative value */
    if (label_vector->offset == 0)
	++label_vector->offset;

    if ((offset = label_vector->offset) >= label_vector->length) {
	erenew_vector(label_vector, label_vector->length + 32);
	labels = label_vector->ptr;
    }
    offset = -offset;
    label->value = efixnum(offset);
    ++label_vector->offset;

    return (-offset);
}

static void
record_forward_patch(eword_t offset, eword_t label)
{
    if (patch_vector->offset >= patch_vector->length) {
	erenew_vector(patch_vector, patch_vector->length + 32);
	patches = patch_vector->ptr;
    }
    patches[patch_vector->offset++] = offset;
    patches[patch_vector->offset++] = label;
}

static void
emit_branch_forward_patch(ecode_t code, eword_t label)
{
    eobject_t		offset;

    switch (code) {
	case eop_jt:
	    offset = jit_bnei_l(jit_forward(), JIT_R0, 0);
	    break;
	case eop_jf:
	    offset = jit_beqi_l(jit_forward(), JIT_R0, 0);
	    break;
	default:
	    offset = jit_jmpi(jit_forward());
	    break;
    }
    record_forward_patch((eword_t)offset, label);
}

static void
emit_branch_finalize(ecode_t code, elabel_t *label)
{
    eword_t		offset;

    if (label->value == null) {
	offset = allocate_forward_label(label);
	emit_branch_forward_patch(code, offset);
    }
    else {
	offset = eget_fixnum(label->value);
	/* if offset is negative, it means the label is not yet defined */
	if (offset >= 0) {
	    switch (code) {
		case eop_jt:
		    (void)jit_bnei_l(labels[offset], JIT_R0, 0);
		    break;
		case eop_jf:
		    (void)jit_beqi_l(labels[offset], JIT_R0, 0);
		    break;
		default:
		    (void)jit_jmpi(labels[offset]);
		    break;
	    }
	}
	else
	    emit_branch_forward_patch(code, -offset);
    }
}

static void
emit_branch(ecode_t code, elabel_t *label)
{
    etype_t		type;
    eobject_t		label_any;
    eobject_t		label_mpr;
    eobject_t		label_rgr;
    eobject_t		label_not_int;
    eobject_t		label_int_next;
    eobject_t		label_not_float;
    eobject_t		label_float_next;
#if __WORDSIZE == 32
    eobject_t		label_int_not_zero;
#endif

    switch (lit_type) {
	case t_int8:			case t_uint8:
	case t_int16:			case t_uint16:
	case t_int32:			case t_uint32:
	    type = t_int32;
	    break;
	case t_int:
	    /* t_uint may coerce to mpz_t in emit_flush() */
	    type = t_int;
	    break;
	case t_float32:
	    if (!(lit_sync & lit_sync_float_ext))
		jit_extr_f_d(JIT_FPR0, JIT_FPR1);
	case t_float:
	    type = t_float;
	    break;
	default:
	    type = t_any;
	    break;
    }

    emit_flush(0);

    if (code != eop_j) {
	label_not_int = label_int_next = null;
	label_not_float = label_float_next = null;
#if __WORDSIZE == 32
	label_int_not_zero = null;
#endif

	if (type == t_any) {
	    jit_ldxi_i(JIT_R0, JIT_V0,
		       offsetof(ethread_t, r0) + offsetof(evalue_t, t));
	    label_not_int = jit_bnei_l(jit_forward(), JIT_R0, t_int);
	}

	if (type == t_any || type == t_int32 || type == t_int) {
#if __WORDSIZE == 32
	    if (type == t_any)
		jit_ldxi_i(JIT_R0, JIT_V0,
			   offsetof(ethread_t, r0) +
			   offsetof(evalue_t, v.i) +
			   offsetof(edata_t, s.l));
	    else
		jit_movr_i(JIT_R0, JIT_V1);
	    /* short circuit if low 32 bits is not zero */
	    if (code == eop_jt)
		emit_branch_finalize(eop_jt, label);
	    if (type != t_int32) {
		if (code == eop_jf)
		    label_int_not_zero = jit_bnei_l(jit_forward(), JIT_R0, 0);
		if (type == t_any)
		    jit_ldxi_i(JIT_R0, JIT_V0,
			       offsetof(ethread_t, r0) +
			       offsetof(evalue_t, v.i) +
			       offsetof(edata_t, s.h));
		else
		    jit_movr_i(JIT_R0, JIT_V2);
	    }
	    if (code == eop_jf || type != t_int32)
		emit_branch_finalize(code, label);
#else
	    if (type == t_any)
		jit_ldxi_l(JIT_R0, JIT_V0,
			   offsetof(ethread_t, r0) + offsetof(evalue_t, v.i));
	    else
		jit_movr_l(JIT_R0, JIT_V1);
	    emit_branch_finalize(code, label);
#endif
	}

	if (type == t_any) {
	    label_int_next = jit_jmpi(jit_forward());
	    jit_patch(label_not_int);
	    label_not_float = jit_bnei_l(jit_forward(), JIT_R0, t_float);
	}

	if (type == t_any || type == t_float) {
#if defined(__i386__) && HAVE_SIMD
#  if !defined(jit_sse2_p)
#    define jit_sse2_p()		0
#  endif
	    if (!jit_sse2_p()) {
		/* if SSE vector instructions have been used, the
		 * alternate zero compare (using i387) may go crazy */
		(void)jit_calli(evm_float_zero_p);
		jit_eqi_l(JIT_R0, JIT_RET, 0);
	    }
	    else {
#endif
		if (type != t_float)
		    jit_ldxi_d(JIT_FPR0, JIT_V0,
			       offsetof(ethread_t, r0) +
			       offsetof(evalue_t, v.d));
		jit_movi_d(JIT_FPR1, 0.0);
		jit_ltgtr_d(JIT_R0, JIT_FPR0, JIT_FPR1);
		emit_branch_finalize(code, label);
#if defined(__i386__) && HAVE_SIMD
	    }
#endif
	    emit_branch_finalize(code, label);
	}

	if (type == t_any) {
	    label_float_next = jit_jmpi(jit_forward());

	    jit_patch(label_not_float);
	    label_mpr = jit_beqi_l(jit_forward(), JIT_R0, t_mpr);
	    label_rgr = jit_beqi_l(jit_forward(), JIT_R0, t_rgr);
	    label_any = jit_bnei_l(jit_forward(), JIT_R0, t_shr);
	    jit_patch(label_mpr);
	    jit_patch(label_rgr);
	    jit_prepare(1);
	    jit_ldxi_p(JIT_R0, JIT_V0,
		       offsetof(ethread_t, r0) + offsetof(evalue_t, v.o));
	    jit_pusharg_p(JIT_R0);
	    jit_finish(mpfr_zero_p);
	    jit_eqi_l(JIT_R0, JIT_RET, 0);

	    /* label_any means all other objects that can hold a false (zero)
	     * value have been tested, and now can just use the type field
	     * (already in JIT_R0) to check for the false value, as only null
	     * has the t_void (zero) type */
	    jit_patch(label_any);
	    emit_branch_finalize(code, label);

	    jit_patch(label_int_next);
	    jit_patch(label_float_next);
	}

#if __WORDSIZE == 32
	if (code == eop_jf && type != t_int32)
	    jit_patch(label_int_not_zero);
#endif
    }
    else
	emit_branch_finalize(eop_j, label);
}

static ecode_t
switches(ehash_t *hash, eint32_t offset)
{
    eint32_t		 i;
    ecode_t		 code;
    east_node_t		*node;
    eint32_t		 size;
    eentry_t		*entry;
    eint32_t		*array;
    ecase_t		*cases;
    eint32_t		 index;
    eint32_t		 length;
    eint32_t		 clashes;
    eint_t		 integer;
    evswitch_t		*vswitch;
    ehswitch_t		*hswitch;
    elabel_t		*fallback;
    eint_t		 imin, imax;

    /* default address */
    entry = eget_hash(hash, null);
    assert(entry != null);

    /* fallback is used to fill vswitch unused entries, so that it is
     * not required to check for an invalid 0 distance jump at runtime */
    node = entry->value;
    fallback = node->clabel.label;
    /* remove the 'default' entry as it is no longer used */
    erem_hash(hash, entry);

    /* hash->count is guaranteed to not be zero */
    size = hash->count;
    /* if size is not already a power of two */
    if (size & (size - 1)) {
	for (i = 1 << 30; i >= 0; i >>= 1)
	    if (size & i)
		break;
	size = i << 1;
	/* should only assert on integer wrap */
	assert(size >= hash->count);
    }

    v_check(1);
    enew_object((eobject_t *)&array, t_void,
		(size / 32 + 1) * sizeof(eint32_t));
    v_push(array);

    /* set all bits and simplify code below */
    size -= 1;

    /* check min/max values, and if there are any clashes */
    clashes = 0;
    imax = 0x8000000000000000L;
    imin = 0x7fffffffffffffffL;
    for (i = 0; i < hash->size; i++) {
	for (entry = hash->entries[i]; entry; entry = entry->next) {
	    integer = eget_integer(entry->name);
	    if (integer > imax)
		imax = integer;
	    if (integer < imin)
		imin = integer;
	    integer &= size;
	    if (array[integer / 32] & (1 << (integer & 31)))
		++clashes;
	    else
		array[integer / 32] |= 1 << (integer & 31);
	}
    }

    if (clashes == 0 && imax - imin <= size) {
	/* faster case */
	enew_object((eobject_t *)&vswitch, t_void,
		    sizeof(evswitch_t) + sizeof(eobject_t) * size);
	v_top() = vswitch;
	vswitch->imin = imin;
	vswitch->imax = imax;
	vswitch->mask = size;
	/* dont need to check for a zero distance jump at runtime */
	for (i = 0; i <= size; i++)
	    vswitch->cases[i] = fallback;
	for (i = 0; i < hash->size; i++) {
	    for (entry = hash->entries[i]; entry; entry = entry->next) {
		integer = eget_integer(entry->name);
		index = integer & size;
		node = entry->value;
		vswitch->cases[index] = node->clabel.label;
	    }
	}
	code = eop_jv;
	els[offset] = vswitch;
    }
    else {
	/* slower case */
	enew_object((eobject_t *)&hswitch, t_switch, sizeof(ehswitch_t) +
		    sizeof(eobject_t) * size);
	v_top() = hswitch;
	hswitch->imin = imin;
	hswitch->imax = imax;
	hswitch->mask = size;
	for (i = 0; i < hash->size; i++) {
	    for (entry = hash->entries[i]; entry; entry = entry->next) {
		integer = eget_integer(entry->name);
		index = integer & size;
		if ((cases = hswitch->cases[index]) == null) {
		    enew_object((eobject_t *)&cases, t_void, sizeof(ecase_t));
		    hswitch->cases[index] = cases;
		    length = 0;
		}
		else {
		    /* a clash */
		    length = eobject_length(cases) + sizeof(ecase_t);
		    eresize_object((eobject_t *)(hswitch->cases + index),
				   length);
		    cases = hswitch->cases[index];
		    length = length / sizeof(ecase_t) - 1;
		}
		cases[length].value = integer;
		node = entry->value;
		cases[length].label = node->clabel.label;
	    }
	}
	code = eop_jh;
	els[offset] = hswitch;
    }
    v_dec();

    return (code);
}

static void
record_table_patch(eword_t offset)
{
    if (table_vector->offset >= table_vector->length) {
	erenew_vector(table_vector, table_vector->length + 32);
	tables = table_vector->ptr;
    }
    tables[table_vector->offset++] = offset;
}

static void
emit_table_branch(ehash_t *hash, eint32_t offset)
{
    eobject_t		 insn;
    east_node_t		*node;
    eentry_t		*entry;
    elabel_t		*label;
    eword_t		 index;

    /* reference labels */
    for (index = 0; index < hash->size; index++) {
	for (entry = hash->entries[index]; entry; entry = entry->next) {
	    node = entry->value;
	    label = node->clabel.label;
	    /* should always be null */
	    if (label->value == null)
		(void)allocate_forward_label(label);
	}
    }

    /* call function to choose jump address */
    jit_prepare(1);
    jit_movi_i(JIT_R0, offset);
    jit_pusharg_i(JIT_R0);
    if (switches(hash, offset) == eop_jv)
	jit_finish(evm_jv);
    else
	jit_finish(evm_jh);
    jit_retval_p(JIT_R0);

    /* return is null if no entry found in table, what will cause
     * it to reach a jump to either default entry, or outside of
     * switch, that is the instruction following eop_jh */
    insn = jit_beqi_p(jit_forward(), JIT_R0, 0);
    jit_jmpr(JIT_R0);
    jit_patch(insn);

    record_table_patch(offset);
}

static void
vswitch_patch(evswitch_t *table)
{
    elabel_t	*label;
    eint32_t	 length;
    eint32_t	 offset;

    length = (eobject_length(table) -
	      offsetof(evswitch_t, cases)) / sizeof(eobject_t);
    for (offset = 0; offset < length; offset++) {
	label = table->cases[offset];
	table->cases[offset] = labels[eget_fixnum(label->value)];
    }
}

static void
hswitch_patch(ehswitch_t *table)
{
    ecase_t	*acase;
    elabel_t	*label;
    eint32_t	 count;
    eint32_t	 length;
    eint32_t	 offset;

    count = (eobject_length(table) -
	     offsetof(ehswitch_t, cases)) / sizeof(eobject_t);
    for (offset = 0; offset < count; offset++) {
	acase = table->cases[offset];
	length = eobject_length(acase) / sizeof(ecase_t);
	for (; length; length--, acase++) {
	    label = acase->label;
	    acase->label = labels[eget_fixnum(label->value)];
	}
    }
}

static void
table_patch(eobject_t object)
{
    if (etype(object) == t_switch)
	hswitch_patch(object);
    else
	vswitch_patch(object);
}

static void
record_forward_call(eword_t offset)
{
    if (call_vector->offset >= call_vector->length) {
	erenew_vector(call_vector, call_vector->length + 32);
	calls = call_vector->ptr;
    }
    calls[call_vector->offset++] = jit_jmpi(jit_forward());
    calls[call_vector->offset++] = (eobject_t)offset;
}

static eobject_t
emit_frame(void)
{
    eobject_t	address;

    /* return address */
    address = jit_movi_p(JIT_R2, jit_forward());

    /* sp += 2 */
    jit_ldxi_p(JIT_R1, JIT_V0, offsetof(ethread_t, esp));
    jit_addi_p(JIT_R0, JIT_R1, sizeof(evalue_t) * 2);
    jit_stxi_p(offsetof(ethread_t, esp), JIT_V0, JIT_R0);

    /* FIXME currently vm stack slots are quite huge due to
     * having place for type information, and also storing
     * a complex double by value; if this is not changed,
     * can store two pointers in a vm stack slot, i.e. return
     * address and saved bp pointer
     */

    /* sp[-2].t = sp[-1].t = t_void */
    jit_movi_i(JIT_R0, t_void);
    jit_stxi_i(sizeof(evalue_t) + offsetof(evalue_t, t), JIT_R1, JIT_R0);
    jit_stxi_i(offsetof(evalue_t, t), JIT_R1, JIT_R0);
    /* sp[-2].v.o = return address */
    jit_stxi_p(offsetof(evalue_t, v.o), JIT_R1, JIT_R2);

    /* sp[-1].v.o = bp */
    jit_ldxi_p(JIT_R0, JIT_V0, offsetof(ethread_t, ebp));
    jit_stxi_p(sizeof(evalue_t) + offsetof(evalue_t, v.o), JIT_R1, JIT_R0);

    /* bp = sp - 1 */
    jit_addi_p(JIT_R0, JIT_R1, sizeof(evalue_t));
    jit_stxi_p(offsetof(ethread_t, ebp), JIT_V0, JIT_R0);

    return (address);
}

static void
emit_call(eint32_t offset)
{
    eobject_t	address;

    /* save stack frame */
    address = emit_frame();

    /* call function as a jump - FIXME probably should
     * be converted to real function calls, with stack
     * housekeeping, so that ret* can actually return,
     * instead of doing a possibly processor staling/unpredictable
     * indirect jump */
    if (ejit_pointer_p(ecs[offset]))
	(void)jit_jmpi(ecs[offset]);
    else
	record_forward_call(offset);

    /* where ret* will land */
    jit_patch_movi(address, jit_get_label());
}

static void
emit_method_call(eint32_t offset, eint32_t type)
{
    eobject_t	label_next;
    eobject_t	label_class;
    eobject_t	label_method;
    eobject_t	label_parent;

    /* if (r0.t >= ets_idx) goto label_class; */
    jit_ldxi_i(JIT_R0, JIT_V0,
	       offsetof(ethread_t, r0) + offsetof(evalue_t, t));
    label_class = jit_bgei_ui(jit_forward(), JIT_R0, ets_idx);

    /* V0 = ets + r0.t */
    (void)jit_movi_p(JIT_V1, ets);
    jit_muli_i(JIT_R0, JIT_R0, sizeof(ertti_t));
    jit_addr_p(JIT_V1, JIT_V1, JIT_R0);
    /* if (ets[r0.t].nmethod <= offset) goto label_method; */
    jit_ldxi_i(JIT_R0, JIT_V1, offsetof(ertti_t, nmethod));
    label_method = jit_blei_ui(jit_forward(), JIT_R0, offset);

    /* if (ets[r0.t].methods[offset].type != type) goto label_parent; */
    jit_ldxi_p(JIT_V1, JIT_V1, offsetof(ertti_t, methods));
    jit_ldxi_i(JIT_R0, JIT_V1, offset * sizeof(emethod_t) +
	       offsetof(emethod_t, type));
    label_parent = jit_bnei_ui(jit_forward(), JIT_R0, type);

    /* save stack frame - clobbers r0, r1 and r2 */
    label_next = emit_frame();

    /* R0 = ets[r0.t].methods[atom->offset].code */
    jit_ldxi_p(JIT_R0, JIT_V1, offset * sizeof(emethod_t) +
	       offsetof(emethod_t, code));
    /* indirect call to virtual method associated with r0.t type */
    jit_jmpr(JIT_R0);

    /* error handling */
    jit_patch(label_class);
    jit_patch(label_method);
    jit_patch(label_parent);
    call_v_i(evm_raise, except_type_mismatch);

    /* where ret* will land */
    jit_patch_movi(label_next, jit_get_label());
}

static void
record_forward_method(eword_t type, eword_t offset)
{
    eobject_t		object;
    if (method_vector->offset + 3 >= method_vector->length) {
	erenew_vector(method_vector, method_vector->length + 32);
	methods = method_vector->ptr;
    }
    object = jit_jmpi(jit_forward());
    methods[method_vector->offset++] = (eword_t)object;
    methods[method_vector->offset++] = type;
    methods[method_vector->offset++] = offset;
}

static void
emit_explicit_call(eint32_t offset, eint32_t type)
{
    eobject_t	label_next;
    eobject_t	label_loop;
    eobject_t	label_call;
    eobject_t	label_class;
    eobject_t	label_inherit;

    /* if (r0.t >= ets_idx) goto label_class; */
    jit_ldxi_i(JIT_R0, JIT_V0,
	       offsetof(ethread_t, r0) + offsetof(evalue_t, t));
    label_class = jit_bgei_ui(jit_forward(), JIT_R0, ets_idx);

    (void)jit_movi_p(JIT_V1, ets);

    /* label_loop: */
    label_loop = jit_get_label();
    /* if (R0 == type) goto label_call; */
    label_call = jit_beqi_i(jit_forward(), JIT_R0, type);
    /* if (R0 == t_void) goto label_inherit; */
    label_inherit = jit_beqi_i(jit_forward(), JIT_R0, t_void);
    /* R0 = ets[R0].super */
    jit_muli_i(JIT_R0, JIT_R0, sizeof(ertti_t));
    jit_addi_i(JIT_R0, JIT_R0, offsetof(ertti_t, super));
    jit_ldxr_i(JIT_R0, JIT_V1, JIT_R0);
    /* goto label_loop */
    (void)jit_jmpi(label_loop);

    /* call explicit method */
    jit_patch(label_call);

    /* save stack frame */
    label_next = emit_frame();

    if (ejit_pointer_p(ets[type].methods[offset].code))
	(void)jit_jmpi(ets[type].methods[offset].code);
    else
	record_forward_method(type, offset);

    /* error handling */
    jit_patch(label_class);
    jit_patch(label_inherit);
    call_v_i(evm_raise, except_type_mismatch);

    /* where ret* will land */
    jit_patch_movi(label_next, jit_get_label());
}

static void
emit_enter(east_node_t *node)
{
    eint32_t	 length;
    eint32_t	 offset;
    eint32_t	*vector;

    /* FIXME only needs to coerce arguments if called via apply as in other
     * cases it is already correct; so far apply allows pointers to different
     * function types; only requirement is matching number of arguments.
     */

    if ((length = node->center.acount + node->center.lcount)) {
	/* FIXME				FIXME
	 * FIXME  Cheat gc, rules, etc...	FIXME
	 * FIXME				FIXME */
	vector = (eint32_t *)malloc(length * sizeof(eint32_t));
	for (offset = 0; offset < length; offset++)
	    vector[offset] = node->center.types[offset];
    }
    else
	vector = null;
    jit_prepare(5);
    if (vector) {
	(void)jit_movi_p(JIT_R0, vector);
	jit_pusharg_p(JIT_R0);
	jit_movi_i(JIT_R0, node->center.lcount);
    }
    else {
	jit_movi_ul(JIT_R0, 0);
	jit_pusharg_ul(JIT_R0);
    }
    jit_pusharg_i(JIT_R0);
    if (node->center.acount != node->center.lcount)
	jit_movi_i(JIT_R0, node->center.acount);
    jit_pusharg_i(JIT_R0);
    if (node->center.extra != node->center.acount)
	jit_movi_i(JIT_R0, node->center.extra);
    jit_pusharg_i(JIT_R0);
    if (node->center.local != node->center.extra)
	jit_movi_i(JIT_R0, node->center.local);
    jit_pusharg_i(JIT_R0);
    jit_finish(evm_enter);
}

static void
emit_vnewi(east_node_t *node)
{
    eint32_t	 offset;
    eint32_t	*vector;

    /* FIXME				FIXME
     * FIXME  Cheat gc, rules, etc...	FIXME
     * FIXME				FIXME */
    vector = malloc(node->cvector.rank * sizeof(eint32_t));
    for (offset = 0; offset < node->cvector.rank; offset++)
	vector[offset] = node->cvector.dims[offset];
    jit_prepare(4);
    (void)jit_movi_p(JIT_R0, vector);
    jit_pusharg_p(JIT_R0);
    jit_movi_i(JIT_R0, node->cvector.rank);
    jit_pusharg_i(JIT_R0);
    if (node->cvector.length != node->cvector.rank)
	jit_movi_i(JIT_R0, node->cvector.length);
    jit_pusharg_i(JIT_R0);
    if (node->cvector.type != node->cvector.length)
	jit_movi_i(JIT_R0, node->cvector.type);
    jit_pusharg_i(JIT_R0);
    jit_finish(evm_vnewi);
}

static void
emit_try(east_list_t *list)
{
    east_node_t		*node;
    eobject_t		 label_protect;
    eobject_t		 label_address;

    /* pass as argument the offset after the jump to catch table */
    jit_prepare(1);
    label_address = jit_movi_p(JIT_R0, jit_forward());
    jit_pusharg_p(JIT_R0);
    jit_finish(evm_try);

    /* jump to start of protected code */
    label_protect = jit_jmpi(jit_forward());

    assert(list != null);
    node = east_node(list);
    assert(node->code == eop_j);

    /* arrival on exception (evm_try argument) */
    jit_patch_movi(label_address, jit_get_label());

#if !JIT_REENTER && !JIT_SETJMP
    /* reload thread_self, that may have been overwritten
     * if coming from siglongjmp() */
    (void)jit_calli(get_thread_self);
    jit_retval_p(JIT_V0);
#endif

    /* generate jump to the catch handler */
    emit_branch(eop_j, node->clabel.label);

    /* start protected code */
    jit_patch(label_protect);
}

static void
emit_stackref(eint32_t offset, eint32_t type)
{
    /* r0.t = type */
    jit_movi_i(JIT_R0, type);
    jit_stxi_i(offsetof(ethread_t, r0) + offsetof(evalue_t, t), JIT_V0, JIT_R0);
    /* r0.v.o = bp + offset */
    jit_ldxi_p(JIT_R0, JIT_V0, offsetof(ethread_t, ebp));
    jit_addi_p(JIT_R0, JIT_R0, offset * sizeof(evalue_t));
    jit_stxi_p(offsetof(ethread_t, r0) + offsetof(evalue_t, v.o),
	       JIT_V0, JIT_R0);
}

#define CODE		1
#include "ethunder.c"
#include "erain.c"
#undef CODE

void
elightning(efunction_t *function)
{
    ecdd_t		 cdd;
    eobject_t		 base;
    east_node_t		*node;
    east_list_t		*list;
    evector_t		*name;
    enote_t		*note;
    eint32_t		 type;
    edata_t		 data;
    elabel_t		*label;
    eobject_t		 object;
    eint32_t		 offset;

    if (cfg_verbose > 3)
	ewrite_code(function->code);

    /* reset function scope information */
    label_vector->offset = patch_vector->offset = table_vector->offset = 0;

    /* remember this function is defined */
    base = jit_get_label();
    if (function->record == null) {
	offset = function->name ? function->name->offset : 0;
	ecs[offset] = base;
    }
    else {
	type = function->record->type;
	offset = function->name->offset;
	assert((euint32_t)type < ets_idx);
	ets[type].methods[offset].code = base;
    }

    name = null;
    enew_note((eobject_t *)&note, (eint8_t *)base,
	      function->name ? function->name->name : null);

    /* fake initial value, safe because addresses only grow (at least until
     * the base address need to be remaped, but notes should be trivial to
     * relocate) */
    eset_note(note, 16 * 1024 * 1024 - ((eint8_t *)base - (eint8_t *)the_code));

    lit_type = t_any;
    for (list = function->code; list; list = east_next(list)) {
	node = east_node(list);

	switch (node->code) {

	    case eop_label:
		/* *must* call emit_flush and not emit_clear as there may
		 * be code that loads an implicit value just before a label,
		 * what is a common expansion of the ternary operator */
		emit_flush(-stack_vector->offset);
		label = node->clabel.label;
		if (label->value) {
		    /* if label already referenced */
		    offset = eget_fixnum(label->value);
		    /* ensure forward label is actually undefined */
		    assert(offset < 0);
		    offset = -offset;
		    label->value = efixnum(offset);
		    labels[offset] = jit_get_label();
		}
		else {
		    /* remember label address is known */
		    if (label_vector->offset >= label_vector->length) {
			erenew_vector(label_vector, label_vector->length + 32);
			labels = label_vector->ptr;
		    }
		    label->value = efixnum(label_vector->offset);
		    labels[label_vector->offset++] = jit_get_label();
		}
		break;

	    case eop_nil:
		emit_clear(0);
		jit_movi_l(JIT_R0, t_void);
		jit_stxi_i(offsetof(ethread_t, r0) + offsetof(evalue_t, t),
			   JIT_V0, JIT_R0);
		jit_stxi_p(offsetof(ethread_t, r0) + offsetof(evalue_t, v.o),
			   JIT_V0, JIT_R0);
		break;

	    case eop_int:
		list = emit_int(list, node);
		break;

	    case eop_long:
		list = emit_long(list, node);
		break;

	    case eop_float:
		list = emit_float(list, node);
		break;

	    case eop_ll:
		emit_clear(0);
		/* constants are not relocated */
		data.P = els[immediate_offset(node)];
		type = eobject_type(data.P);
		jit_movi_i(JIT_R0, type);
		jit_stxi_i(offsetof(ethread_t, r0) + offsetof(evalue_t, t),
			   JIT_V0, JIT_R0);
		if (type != t_cdd) {
		    jit_movi_p(JIT_R0, data.P);
		    jit_stxi_p(offsetof(ethread_t, r0) +
			       offsetof(evalue_t, v.o), JIT_V0, JIT_R0);
		}
		else {
		    cdd = *(ecdd_t *)data.P;
		    data.F = real(cdd);
		    jit_movi_d(JIT_FPR0, data.F);
		    jit_stxi_d(offsetof(ethread_t, r0) +
			       offsetof(evalue_t, v.d), JIT_V0, JIT_FPR0);
		    if (data.F != imag(cdd)) {
			data.F = imag(cdd);
			jit_movi_d(JIT_FPR0, data.F);
		    }
		    jit_stxi_d(offsetof(ethread_t, r0) +
			       offsetof(evalue_t, v.d) + sizeof(double),
			       JIT_V0, JIT_FPR0);
		}
		break;

	    case eop_ld:
		emit_clear(0);
		call_v_i(evm_ld, node->csymbol.symbol->offset);
		break;

	    case eop_ldt:
		list = emit_ldt(list, node);
		break;

	    case eop_lb:
		list = emit_lb(list, node);
		break;

	    case eop_lbt:
		list = emit_lbt(list, node);
		break;

	    case eop_ls:
		emit_clear(0);
		call_v_i(evm_ls, node->cstack.offset);
		break;

	    case eop_lh:
		emit_flush(-1);
		call_v_v(evm_lh);
		break;

	    case eop_lhref:
		emit_flush(node->chash.arg != 0);
		call_v_i(evm_lhref, node->chash.arg);
		break;

	    case eop_lv:
		emit_flush(-node->cvector.rank);
		call_v_i(evm_lv, node->cvector.rank);
		/* FIXME currently typed vectors actually change type,
		 * but must be modified */
		break;

	    case eop_lvi:
		emit_flush(0);
		call_v_i(evm_lvi, node->cvector.offset);
		break;

	    case eop_lvref:
		emit_flush(-(node->cvector.rank - !!node->cvector.arg - 1));
		call_v_i_i(evm_lvref, node->cvector.rank, node->cvector.arg);
		break;

	    case eop_lr:
		list = emit_lr(list, node);
		break;

	    case eop_lrref:
		emit_flush(!!node->crecord.arg + 1);
		call_v_i_i_i_i(evm_lrref, node->crecord.record->type,
			       node->crecord.symbol->offset,
			       node->crecord.symbol->type,
			       node->crecord.arg);
		break;

	    case eop_lc:
		list = emit_lc(list, node);
		break;

	    case eop_lref:
		emit_flush(0);
		call_v_v(evm_lref);
		break;

	    case eop_pd:
		emit_clear(0);
		/* r0.t = t_globalref */
		jit_movi_i(JIT_R0, t_globalref);
		jit_stxi_i(offsetof(ethread_t, r0) * offsetof(evalue_t, t),
			   JIT_V0, JIT_R0);
		/* r0.v.o = eds + atom->offset */
		(void)jit_movi_p(JIT_R0, (eint8_t *)eds +
				 node->csymbol.symbol->offset *
				 sizeof(eobject_t));
		jit_stxi_p(offsetof(ethread_t, r0) + offsetof(evalue_t, v.o),
			   JIT_V0, JIT_R0);
		break;

	    case eop_pdt:
		emit_clear(0);
		call_v_i(evm_pdt, node->csymbol.symbol->offset);
		break;

	    case eop_pb:
		emit_clear(0);
		emit_stackref(node->csymbol.symbol->offset, t_stackref);
		break;

	    case eop_pbt:
		emit_clear(0);
		emit_stackref(node->csymbol.symbol->offset, t_t_stackref);
		break;

	    case eop_pv:
		emit_flush(-node->cvector.rank);
		call_v_i(evm_pv, node->cvector.rank);
		break;

	    case eop_pvi:
		emit_flush(0);
		call_v_i(evm_pvi, node->cvector.offset);
		break;

	    case eop_pr:
		emit_flush(0);
		call_v_i_i_i(evm_pr, node->crecord.record->type,
			     node->crecord.symbol->offset,
			     node->crecord.symbol->type);
		break;

	    case eop_pp:
		emit_flush(0);
		call_v_v(evm_pp);
		break;

	    case eop_anon:
		emit_clear(0);
		call_v_i_i(evm_anon, node->crecord.record->type,
			   node->crecord.record->type ?
			   node->crecord.record->size : sizeof(eobject_t));
		break;

	    case eop_sd:
		emit_sync(0);
		call_v_i(evm_sd, node->csymbol.symbol->offset);
		break;

	    case eop_sdt:
		list = emit_sdt(list, node);
		break;

	    case eop_sb:
		list = emit_sb(list, node);
		break;

	    case eop_sbt:
		list = emit_sbt(list, node);
		break;

	    case eop_ss:
		list = emit_ss(list, node);
		break;

	    case eop_sst:
		list = emit_sst(list, node);
		break;

	    case eop_ssv:
		list = emit_ssv(list, node);
		break;

	    case eop_sh:
		emit_sync(-2);
		call_v_v(evm_sh);
		break;

	    case eop_shref:
		emit_sync(-(!!node->chash.arg + 1));
		call_v_i(evm_shref, node->chash.arg);
		break;

	    case eop_sv:
		emit_sync(-(node->cvector.rank + 1));
		call_v_i(evm_sv, node->cvector.rank);
		break;

	    case eop_svi:
		emit_sync(-1);
		call_v_i(evm_svi, node->cvector.offset);
		break;

	    case eop_svref:
		emit_sync(-(!!node->cvector.arg + 1));
		call_v_i(evm_svref, node->cvector.arg);
		break;

	    case eop_sr:
		list = emit_sr(list, node);
		break;

	    case eop_srref:
		emit_sync(-(!!node->crecord.arg + 1));
		call_v_i(evm_srref, node->crecord.arg);
		break;

	    case eop_sc:
		list = emit_sc(list, node);
		break;

	    case eop_sref:
		emit_sync(-1);
		call_v_v(evm_sref);
		break;

	    case eop_jt:
		emit_branch(eop_jt, node->clabel.label);
		break;

	    case eop_jf:
		emit_branch(eop_jf, node->clabel.label);
		break;

	    case eop_j:
		emit_branch(eop_j, node->clabel.label);
		break;

	    case eop_jh:
		emit_flush(0);
		/* eop_jv not generated at this point */
		emit_table_branch(els[immediate_offset(node)],
				  immediate_offset(node));
		break;

	    case eop_enter:
		emit_flush(0);
		emit_enter(node);
		break;

	    case eop_begin:
		emit_flush(node->cstack.length);
		call_v_i(evm_begin, node->cstack.length);
		break;

	    case eop_off:
		emit_sync(1);
		call_v_i(evm_off, node->cvector.rank);
		break;

	    case eop_push:
		list = emit_push(list, node);
		break;

	    case eop_pusht:
		list = emit_pusht(list, node);
		break;

	    case eop_pushv:
#if 0
		/* FIXME jit generation time causes a significant delay
		 * when executing check/math.e due to too many evm_pushv
		 * calls, that clearly does not pay in the slight runtime
		 * speed increase for the values that are actually ints
		 * or floats (delay should be gc due to memory allocations) */
		emit_sync(1);
		call_v_v(evm_pushv);
#else
		list = emit_pushv(list, node);
#endif
		break;

	    case eop_pop:
		emit_clear(-1);
		call_v_v(evm_pop);
		break;

	    case eop_call:
		emit_clear(-node->capply.length);
		emit_call(node->capply.offset);
		/* FIXME check return type */
		break;

	    case eop_meth:
		emit_flush(-node->capply.length);
		emit_method_call(node->capply.offset, node->capply.type);
		break;

	    case eop_ecm:
		emit_flush(-node->capply.length);
		emit_explicit_call(node->capply.offset, node->capply.type);
		break;

	    case eop_blt:
		emit_clear(-node->capply.length);
		call_v_i_i(evm_blt, node->capply.offset, node->capply.length);
		/* builtins (currently) have untyped return value */
		break;

	    case eop_ret:
		emit_flush(0);
		call_p_i(evm_ret, 1);
		jit_retval_p(JIT_R0);
		jit_jmpr(JIT_R0);
		break;

	    case eop_rett:
		emit_flush(0);
		call_p_i_i(evm_rett, 1, node->creturn.type);
		jit_retval_p(JIT_R0);
		jit_jmpr(JIT_R0);
		break;

	    case eop_reti:
		emit_flush(0);
		call_p_i(evm_ret, node->creturn.length);
		jit_retval_p(JIT_R0);
		jit_jmpr(JIT_R0);
		break;

	    case eop_retit:
		emit_flush(0);
		call_p_i_i(evm_rett, node->creturn.length, node->creturn.type);
		jit_retval_p(JIT_R0);
		jit_jmpr(JIT_R0);
		break;

	    case eop_ne:
	    case eop_lt:
	    case eop_le:
	    case eop_eq:
	    case eop_ge:
	    case eop_gt:
		list = emit_cmp(list, node);
		break;

	    case eop_and:
		emit_flush(-1);
		call_v_v(evm_and);
		break;

	    case eop_or:
		emit_flush(-1);
		call_v_v(evm_or);
		break;

	    case eop_xor:
		emit_flush(-1);
		call_v_v(evm_xor);
		break;

	    case eop_mul2:
		emit_flush(-1);
		call_v_v(evm_mul2);
		break;

	    case eop_div2:
		emit_flush(-1);
		call_v_v(evm_div2);
		break;

	    case eop_shl:
		emit_flush(-1);
		call_v_v(evm_shl);
		break;

	    case eop_shr:
		emit_flush(-1);
		call_v_v(evm_shr);
		break;

	    case eop_add:
	    case eop_sub:
		list = emit_addsub(list, node);
		break;

	    case eop_mul:
		emit_flush(-1);
		call_v_v(evm_mul);
		break;

	    case eop_div:
		emit_flush(-1);
		call_v_v(evm_div);
		break;

	    case eop_trunc2:
		emit_flush(-1);
		call_v_v(evm_trunc2);
		break;

	    case eop_rem:
		emit_flush(-1);
		call_v_v(evm_rem);
		break;

	    case eop_atan2:
		emit_flush(-1);
		call_v_v(evm_atan2);
		break;

	    case eop_pow:
		emit_flush(-1);
		call_v_v(evm_pow);
		break;

	    case eop_hypot:
		emit_flush(-1);
		call_v_v(evm_hypot);
		break;

	    case eop_complex:
		emit_flush(-1);
		call_v_v(evm_complex);
		break;

	    case eop_inc:
		list = emit_inc(list, node);
		break;

	    case eop_dec:
		list = emit_dec(list, node);
		break;

	    case eop_bool:
		emit_flush(0);
		call_v_v(evm_bool);
		break;

	    case eop_not:
		emit_flush(0);
		call_v_v(evm_not);
		break;

	    case eop_com:
		emit_flush(0);
		call_v_v(evm_com);
		break;

	    case eop_inv:
		emit_flush(0);
		call_v_v(evm_inv);
		break;

	    case eop_neg:
		emit_flush(0);
		call_v_v(evm_neg);
		break;

	    case eop_signbit:
		emit_flush(0);
		call_v_v(evm_signbit);
		break;

	    case eop_signum:
		emit_flush(0);
		call_v_v(evm_signum);
		break;

	    case eop_rational:
		emit_flush(0);
		call_v_v(evm_rational);
		break;

	    case eop_integer_p:
		emit_flush(0);
		call_v_v(evm_integer_p);
		break;

	    case eop_rational_p:
		emit_flush(0);
		call_v_v(evm_rational_p);
		break;

	    case eop_float_p:
		emit_flush(0);
		call_v_v(evm_float_p);
		break;

	    case eop_real_p:
		emit_flush(0);
		call_v_v(evm_real_p);
		break;

	    case eop_complex_p:
		emit_flush(0);
		call_v_v(evm_complex_p);
		break;

	    case eop_number_p:
		emit_flush(0);
		call_v_v(evm_number_p);
		break;

	    case eop_finite_p:
		emit_flush(0);
		call_v_v(evm_finite_p);
		break;

	    case eop_inf_p:
		emit_flush(0);
		call_v_v(evm_inf_p);
		break;

	    case eop_nan_p:
		emit_flush(0);
		call_v_v(evm_nan_p);
		break;

	    case eop_num:
		emit_flush(0);
		call_v_v(evm_num);
		break;

	    case eop_den:
		emit_flush(0);
		call_v_v(evm_den);
		break;

	    case eop_real:
		emit_flush(0);
		call_v_v(evm_real);
		break;

	    case eop_imag:
		emit_flush(0);
		call_v_v(evm_imag);
		break;

	    case eop_arg:
		emit_flush(0);
		call_v_v(evm_arg);
		break;

	    case eop_conj:
		emit_flush(0);
		call_v_v(evm_conj);
		break;

	    case eop_floor:
		emit_flush(0);
		call_v_v(evm_floor);
		break;

	    case eop_trunc:
		emit_flush(0);
		call_v_v(evm_trunc);
		break;

	    case eop_round:
		emit_flush(0);
		call_v_v(evm_round);
		break;

	    case eop_ceil:
		emit_flush(0);
		call_v_v(evm_ceil);
		break;

	    case eop_abs:
		emit_flush(0);
		call_v_v(evm_abs);
		break;

	    case eop_sqrt:
		emit_flush(0);
		call_v_v(evm_sqrt);
		break;

	    case eop_cbrt:
		emit_flush(0);
		call_v_v(evm_cbrt);
		break;

	    case eop_sin:
		emit_flush(0);
		call_v_v(evm_sin);
		break;

	    case eop_cos:
		emit_flush(0);
		call_v_v(evm_cos);
		break;

	    case eop_tan:
		emit_flush(0);
		call_v_v(evm_tan);
		break;

	    case eop_asin:
		emit_flush(0);
		call_v_v(evm_asin);
		break;

	    case eop_acos:
		emit_flush(0);
		call_v_v(evm_acos);
		break;

	    case eop_atan:
		emit_flush(0);
		call_v_v(evm_atan);
		break;

	    case eop_sinh:
		emit_flush(0);
		call_v_v(evm_sinh);
		break;

	    case eop_cosh:
		emit_flush(0);
		call_v_v(evm_cosh);
		break;

	    case eop_tanh:
		emit_flush(0);
		call_v_v(evm_tanh);
		break;

	    case eop_asinh:
		emit_flush(0);
		call_v_v(evm_asinh);
		break;

	    case eop_acosh:
		emit_flush(0);
		call_v_v(evm_acosh);
		break;

	    case eop_atanh:
		emit_flush(0);
		call_v_v(evm_atanh);
		break;

	    case eop_proj:
		emit_flush(0);
		call_v_v(evm_proj);
		break;

	    case eop_exp:
		emit_flush(0);
		call_v_v(evm_exp);
		break;

	    case eop_log:
		emit_flush(0);
		call_v_v(evm_log);
		break;

	    case eop_log2:
		emit_flush(0);
		call_v_v(evm_log2);
		break;

	    case eop_log10:
		emit_flush(0);
		call_v_v(evm_log10);
		break;

	    case eop_new:
		emit_clear(0);
		call_v_i_i(evm_new, node->crecord.record->type,
			   node->crecord.record->size);
		break;

	    case eop_mv:
		emit_flush(-1);
		call_v_i(evm_mv, node->crecord.record->size);
		break;

	    case eop_vnew:
		emit_flush(-(node->cvector.rank - 1));
		call_v_i_i(evm_vnew, node->cvector.type, node->cvector.rank);
		break;

	    case eop_vnewi:
		emit_flush(0);
		emit_vnewi(node);
		break;

	    case eop_vmv:
		emit_flush(-1);
		call_v_i(evm_vmv, node->cvector.length);
		break;

	    case eop_renew:
		emit_flush(-1);
		call_v_v(evm_renew);
		break;

	    case eop_renewi:
		emit_flush(0);
		call_v_i(evm_renewi, node->cvector.length);
		break;

	    case eop_fref:
		emit_clear(0);
		call_v_i_i_i(evm_fref, node->capply.offset,
			     node->capply.length, node->capply.arg);
		break;

	    case eop_apply:
		emit_flush(-node->capply.length);
		/* FIXME only require jumping into the function code
		 * if not a builtin (otherwise, the returned address
		 * is just the address after evm_apply call...) */
		jit_prepare(2);
		object = jit_movi_p(JIT_R0, jit_forward());
		jit_pusharg_p(JIT_R0);
		jit_movi_i(JIT_R0, node->capply.length);
		jit_pusharg_i(JIT_R0);
		jit_finish(evm_apply);
		jit_retval_p(JIT_R0);
		jit_jmpr(JIT_R0);
		/* where ret* will arrive */
		jit_patch_movi(object, jit_get_label());
		break;

	    case eop_aret:
		emit_sync(0);
		call_v_v(evm_aret);
		/* implicit type not modified */
		break;

	    case eop_szof:
		emit_flush(0);
		call_v_v(evm_szof);
		break;

	    case eop_szofva:
		emit_flush(0);
		call_v_v(evm_szofva);
		break;

	    case eop_szofdim:
		emit_flush(-1);
		call_v_v(evm_szofdim);
		break;

	    case eop_szofdimi:
		emit_flush(-1);
		call_v_i(evm_szofdimi, node->cvector.arg);
		break;

	    case eop_tpof:
		emit_flush(0);
		call_v_v(evm_tpof);
		break;

	    case eop_subtpof:
		emit_flush(-1);
		call_v_v(evm_subtpof);
		break;

	    case eop_rankof:
		emit_flush(0);
		call_v_v(evm_rankof);
		break;

	    case eop_vas:
		emit_sync(0);
		call_v_i(evm_vas, node->cstack.length);
		/* implicit type not modified */
		break;

	    case eop_val:
		emit_sync(-node->cstack.length);
		call_v_i(evm_val, node->cstack.length);
		/* implicit type not modified */
		break;

	    case eop_lva:
		emit_flush(0);
		call_v_i(evm_lva, node->cstack.length);
		break;

	    case eop_lvai:
		emit_flush(0);
		call_v_i_i(evm_lvai, node->cstack.length, node->cstack.offset);
		break;

	    case eop_sva:
		emit_sync(-1);
		call_v_i(evm_sva, node->cstack.length);
		/* implicit type not modified */
		break;

	    case eop_svai:
		emit_sync(0);
		call_v_i_i(evm_svai, node->cstack.length, node->cstack.offset);
		/* implicit type not modified */
		break;

	    case eop_pva:
		emit_flush(0);
		call_v_i(evm_pva, node->cstack.length);
		break;

	    case eop_try:
		emit_flush(4);
		emit_try(list = east_next(list));
		break;

	    case eop_catch:
		emit_flush(0);
		call_v_v(evm_catch);
		break;

	    case eop_throw:
		emit_flush(0);
		jit_prepare(1);
		jit_movi_i(JIT_R0, node->cexcept.arg);
		jit_pusharg_i(JIT_R0);
		jit_finish(evm_throw);
		/* jump to catch handler */
		jit_retval_p(JIT_R0);
		jit_jmpr(JIT_R0);
		break;

	    case eop_unwind:
		emit_sync(node->cexcept.arg ? -4 : 0);
		call_v_v(evm_unwind);
		/* implicit type not modified */
		break;

	    case eop_exit:
		emit_clear(0);
		/* if not main thread, this call never returns */
		call_v_v(evm_exit);
		/* if main thread, return from jit */
		(void)jit_movi_i(JIT_RET, 0);
		jit_ret();
		break;

	    case eop_setup:
		emit_sync(0);
		call_v_i_i(evm_setup, node->csymbol.symbol->offset,
			   node->csymbol.symbol->type);
		break;

	    case eop_intp_0:
		emit_flush(0);
		call_v_v(evm_intp_0);
		break;

	    case eop_intp_v:
		emit_flush(0);
		call_v_v(evm_intp_v);
		break;

	    case eop_intp_n1:
		emit_flush(0);
		call_v_v(evm_intp_n1);
		break;

	    case eop_realp_v:
		emit_flush(0);
		call_v_v(evm_realp_v);
		break;

	    case eop_nump_0:
		emit_flush(0);
		call_v_v(evm_nump_0);
		break;

	    case eop_nump_v:
		emit_flush(0);
		call_v_v(evm_nump_v);
		break;

	    case eop_note:
		eannotate(note, node->cnote.name, node->cnote.line,
			  (eint8_t *)jit_get_label() - (eint8_t *)base);
		break;

	    case eop_block:
		/* implicit type not modified as blocks are a special
		 * kind of label, that are known to not be the target
		 * of any jump, as they are actually code following
		 * conditional branches */
		break;

	    default:
		abort();
	}
    }

    /* patch forward jumps */
    for (offset = 0; offset < patch_vector->offset; offset += 2)
	jit_patch_at((eobject_t)patches[offset], labels[patches[offset + 1]]);

    /* patch switches */
    for (offset = 0; offset < table_vector->offset; offset++)
	table_patch(els[tables[offset]]);

#if DISASSEMBLER
    if (cfg_verbose > 1) {
	if (likely(function->name))
	    edisassemble(base, (eint8_t *)jit_get_label() - (eint8_t *)base);
	else
	    edisassemble(the_code,
			 (eint8_t *)jit_get_label() - (eint8_t *)the_code);
    }
#endif

    /* update annotation for this function */
    eset_note(note, (eint8_t *)jit_get_label() - (eint8_t *)base);
    efinish_note(note);

    function->code = null;
}
